﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Data;
using System.Timers;
using System.Threading.Tasks;
using System.Net;
using KC.DataAccess.Global;

namespace KC.Business.Scheduler.ScheduledPlugins
{
	public class ArticleulatorParser : ScheduledPluginBase
	{
		public override void Dispose()
		{
		}
		protected override void CloseInner()
		{
		}
		protected override void FailInner()
		{
		}
		protected async override void RunInner()
		{
			List<DataRow> rows = await GetBodiesToParse();
			Parallel.ForEach(rows, new ParallelOptions() { MaxDegreeOfParallelism = 4}, async r => await ParseBody(r));
		}

		public async Task<List<DataRow>> GetBodiesToParse()
		{
			DataTable dt = await SQL.ExecuteDataTable(ConnStr, "select top 4 * from downloadedpages where result is null");
			List<DataRow> rows = new List<DataRow>();
			foreach (DataRow dr in dt.Rows)
				rows.Add(dr);
			return rows;
		}
		public async Task<string> ConvertDownloadPageRowToPlaintext(DataRow dr)
		{
			if (dr == null) throw new ArgumentNullException("dr");

			// Extract plaintext
			if (!((string)dr["DocText"]).Trim().StartsWith("<")) {
				await SQL.ExecuteNonQuery(ConnStr, "update downloadedpages set result='Invalid XHTML' where ID=" + dr["ID"].ToString());
				return null; }
			if (ArticleulatorSpider.isUrlBanned(dr["URL"].ToString())) {
				await SQL.ExecuteNonQuery(ConnStr, "update downloadedpages set result='Banned' where ID=" + dr["ID"].ToString());
				return null; }
			string result = "";
			string body = ((string)dr["DocText"]).Trim().ToLower();
			if (body.Contains("<body")) body = body.Substring(body.IndexOf(">", body.IndexOf("<body") + 1) + 1);
			if (body.Contains("</body")) body = body.Substring(0, body.IndexOf("</body"));

			// Parse HTML, locate and extract plaintext
			DateTime started = DateTime.Now;
			for (int i = 0; i < body.Length; i++) // skip all HTML and tags' inner contents
				try {
					if (DateTime.Now - started > TimeSpan.FromSeconds(5)) break;
					string sub = body.Substring(i);
					if (sub.StartsWith("<!--")) { i = body.IndexOf("-->", i + 1) + 2; continue; }
					else if (sub.StartsWith("</")) { i = body.IndexOf(">", i + 1); continue; }
					else if (sub.StartsWith("<style") || sub.StartsWith("<a ") || sub.StartsWith("<sup") || sub.StartsWith("<sub")) { i = body.IndexOf(">", body.IndexOf(">", i + 1) + 1); continue; }
					else if (sub.StartsWith("<script")) { i = body.IndexOf("</script>", body.IndexOf(">", i + 1) + 1) + 8; continue; }
					else if (sub.StartsWith("<div") || sub.StartsWith("<p ") || sub.StartsWith("<span")) { i = body.IndexOf(">", i + 1); continue; }
					else if (sub.StartsWith("/>")) { i += 1; continue; }
					else if (sub.StartsWith("/ >")) { i += 1; continue; }
					else if (sub.StartsWith("<")) { i = body.IndexOf(">", i + 1); continue; }
					if (i >= body.Length) break;
					result += body[i]; }
				catch { i++; }
			string stage1 = result;

			// Format result document to facilitate sentence extrapolation and word extraction
			try {
				result = Regex.Replace(result, "\\[(.*)\\]", "");
				result = result.Trim();
				result = System.Web.HttpUtility.HtmlDecode(result);
				result = System.Web.HttpUtility.UrlDecode(result);
				result = result.Replace("\t", " ").Replace("\r", "\n").Replace("\n", ".").Replace("·", " ").Replace("∙", " ").Replace("·", " ").Replace(":", " ");
				while (result.Contains("  ")) result = result.Replace("  ", " ");
				while (result.Contains("..")) result = result.Replace("..", ".");
				while (result.Contains(". ")) result = result.Replace(". ", ".");
				while (result.Contains("..")) result = result.Replace("..", ".");
				while (result.Contains("  ")) result = result.Replace("  ", " ");
				await SQL.ExecuteNonQuery(ConnStr, "update DownloadedPages set FormattedText='" + result.Replace("'", "''") + "', result='Success' where id=" + dr["ID"].ToString());
			}
			catch { SQL.ExecuteNonQuery(ConnStr, "update DownloadedPages set FormattedText='" + result.Replace("'", "''") + "', result='Import Error' where id=" + dr["ID"].ToString()); }

			return result;
		}
		public async Task ParseBody(DataRow dr)
		{
			if (dr == null) throw new ArgumentNullException("dr");
			// Import the plaintext
			try {
				string result = await ConvertDownloadPageRowToPlaintext(dr);
				if (string.IsNullOrEmpty(result)) return;
				await ExtractSentences(result);
				await SQL.ExecuteNonQuery(ConnStr, "update downloadedpages set result='Import Complete' where url='" + dr["URL"].ToString().Replace("'", "''") + "' and (result is null or result='')"); }
			catch { SQL.ExecuteNonQuery(ConnStr, "update downloadedpages set result='Import Error' where url='" + dr["URL"].ToString().Replace("'", "''") + "' and (result is null or result='')"); }
		}
		public async Task<List<string>> ExtractSentences(string plaintext)
		{
			if (string.IsNullOrEmpty(plaintext)) throw new ArgumentNullException("plaintext");
			string[] sentenceMaybes = plaintext.Split('.');
			List<string> sentences = new List<string>();
			double curRow = 0;
			await Task.Run(() => Parallel.ForEach(sentenceMaybes, new ParallelOptions() { MaxDegreeOfParallelism = 4 }, async sentence => {
				curRow++;
				if (sentence.Length < 20) return; // Sentence is too short
				if (sentence.Length > 500) return; // Sentence is too long
				if (sentence.Where(c => c == '(').Count() != sentence.Where(c => c == ')').Count()) return; // Sentence has a mismatched # of parens
				if (sentence.Split(' ').Length < 6) return; // Sentence has too few words
				if ((double)Regex.Matches(sentence.Replace(" ", ""), @"([a-zA-Z\,\;\(\)]{1})", RegexOptions.Singleline).Count / (double)sentence.Replace(" ", "").Length < .7D) return; // Sentence has a low % of letters; likely math

				// Format sentence
				string sentence2 = sentence[0].ToString().ToUpper() + sentence.Substring(1) + ".";
				sentences.Add(sentence2.Trim());
				await SQL.ExecuteNonQuery(ConnStr, "insert into sentences (sentence, source, created) values ('" + sentence2.Replace("'", "''").Trim() + "', 'Extracted', getdate())");

				// import words in this sentence
				foreach (string word in sentence2.Replace(",", "").Replace(";", "").Split(new char[] { ' ' }))
					ImportWord(word); }));
			return sentences;
		}
		public async void ImportWord(string word)
		{
			if (string.IsNullOrEmpty(word) || word.Trim().Length == 0) return;
			string lower = word.ToLower().Replace(".", "").Replace(",", "").Replace(";", "");
			if (!isValidWord(word, lower)) return;
			if (lower.Contains("'")) lower = lower.Substring(0, lower.IndexOf("'"));
			if (lower.Contains("\"") || lower.Contains("=")) return;
			if (lower.Contains("(") || lower.Contains(")")) return;
			if (lower.Contains("<") || lower.Contains(">")) return;
			if (lower.Contains("java")) return;

			if (string.IsNullOrEmpty(lower.Trim())) return;
			if (Regex.Matches(lower, @"(^a-zA-Z\-0-9)").Count > 0) return;
			WordType? wotype = null;
			try { wotype = DetectWordType(ref lower); }
			catch { }
			await SQL.ExecuteNonQuery(ConnStr, "if (select count(id) from words where word='" + lower.Replace("'", "''") + "') = 0 begin insert into words (word, instances, wordtype) values ('" + lower.Replace("'", "''") + "',1, " + (wotype == null ? "NULL" : ("'" + wotype.ToString() + "'")) + ") end " +
									"else begin update words set instances = instances + 1, wordtype=" + (wotype == null ? "NULL" : ("isnull(wordtype,'" + wotype.ToString() + "')")) + " where word='" + lower.Replace("'", "''") + "' end");
		}
		public static bool isValidWord(string word, string lower)
		{
			if (string.IsNullOrEmpty(word)) throw new ArgumentNullException("word");
			if (string.IsNullOrEmpty(lower)) throw new ArgumentNullException("lower");
			string upper = word.ToUpper();

			if (new Regex(@"[^\w\']").Matches(word).Count > 0) return false;
			if (lower.Length == 1 && !words1.Contains(upper)) return false; // all valid 2-letter words
			if (lower.Length == 2 && !words2.Contains(upper)) return false; // all valid 2-letter words
			if (lower.Length == 3 && !words3.Contains(upper)) return false; // all valid 3-letter words

			// Ensure a vowel exists
			bool isVowel = false;
			foreach (char c in lower)
				if (vowels.Contains(c)) isVowel = true;
			if (!isVowel) return false;

			// Ensure a consonant exists
			if (word.Length > 1) { // A, I, and O are valid words
				bool isConsonant = false;
				foreach (char c in lower)
					if (consonants.Contains(c)) isConsonant = true;
				if (!isConsonant) return false;
			}

			if (!upper.Contains("A") && !upper.Contains("E") && !upper.Contains("I") && !upper.Contains("O") && !upper.Contains("U") && !upper.Contains("Y"))
				return false; // all valid words must contain a vowel

			return true;
		}
		private static WordType? DetectWordType(ref string word)
		{
			if (string.IsNullOrEmpty(word)) throw new ArgumentNullException("word");
			string lower = word.ToLower();
			if (adjectives.Contains(lower)) return WordType.Adjective; // "costly" overrides adverb rules
			if (prepositions.Contains(lower)) return WordType.Preposition;
			if (articles.Contains(lower)) return WordType.Article;
			if (adverbs.Contains(lower)) return WordType.Adverb;
			if (conjunctions.Contains(lower)) return WordType.Conjunction;
			if (lower.EndsWith("cancelloaded")) return null;

			// Identify base word
			if (lower.Length < 20 && lower.EndsWith("ing"))
			{ // swing = noun; ching = noun; bling = noun; swinging = verb; chinging = verb; blinging = verb
				string prefix = lower.Substring(0, lower.LastIndexOf("ing"));
				if (prefix.Length > 6 && prefix[prefix.Length - 1] == prefix[prefix.Length - 2]) prefix = prefix.Substring(0, prefix.Length - 1); // if ends with two of the same letter, chop one off
				if (prefix.Length > 3 && prefix.Contains("a") || prefix.Contains("e") || prefix.Contains("i") || prefix.Contains("o") || prefix.Contains("u") || prefix.Contains("y"))
				{
					word = prefix;
					return WordType.Verb;
				}
				else return WordType.Noun;
			}
			if (lower.Length < 20 && lower.EndsWith("ed"))
			{ // tamper/tampered; test/tested; submit/submitted; program/programmed; 
				string prefix = lower.Substring(0, lower.LastIndexOf("ed"));
				if (prefix.Length > 5 && prefix[prefix.Length - 1] == prefix[prefix.Length - 2]) prefix = prefix.Substring(0, prefix.Length - 1); // if ends with two of the same letter, chop one off
				if (prefix.Length > 3 && prefix.EndsWith("i")) prefix = prefix.Substring(0, prefix.Length - 1) + "y"; // specified > specifi > specify
				if (prefix.Length > 2)
				{
					word = prefix;
					return WordType.Verb;
				}
				else return WordType.Noun;
			}


			if (lower.EndsWith("ly")) return WordType.Adverb;

			// First letter capital, all others lower: Noun (generally ~ check other types first in case it starts a sentence)
			if (word.Length > 1 && word[0] == word.ToUpper()[0] && word.Substring(1) == word.ToLower().Substring(1)) return WordType.Noun;

			return null;
		}

		#region Enums and constants

		private const int MaxPageSize = 65536;
		private const int NumParallelism = 2, NumPara = 5, NumSentPerPara = 10;
		public enum SearchEngine { Google }
		private enum WordType { Preposition, Noun, Verb, Adjective, Adverb, Article, Conjunction }
		public enum WordSubtype
		{
			Coordinating, Subordinating, Correlative, Definite, Indefinite, Zero, Simple, Compound, Proper, Common, Countable, Uncountable, Collective, Concrete, Abstract, Transitive, Intransitive, Descriptive, Limiting, Predicate,
			Manner, Place, Time, Frequency, Degree, Personal, Possessive, Demonstrative, Relative, Interrogative
		}

		private static string vowels = "aeiouy";
		private static string consonants = "bcdfghjklmnpqrstvwxyz";
		/// <summary>If true, a URL will not be spidered and downloaded.</summary>
		/// <param name="url">URL to check</param>
		/// <returns>true = Do Not Spider / Download; false = Go Nuts</returns>
		public static bool isUrlBanned(string url)
		{
			if (string.IsNullOrEmpty(url)) throw new ArgumentNullException("url");
			url = url.ToLower().Trim();
			return url.Contains("doi.org")
				|| url.Contains("comcast")
				|| url.Contains(".gov")
				|| url.Contains(".pbs")
				|| url.Contains(".mil")
				|| url.Contains("wiley.com")
				|| (!url.StartsWith("http://") && !url.StartsWith("https://"))
				|| url.Contains("cambridge.")
				|| url.Contains("statcounter.com")
				|| url.Contains("newyorker.com")
				|| url.Contains("google.")
				|| url.Contains("bing.")
				|| url.Contains("@")
				|| url.Contains("yahoo.")
				|| url.Contains("aol.com")
				|| url.Contains("ask.com")
				|| url.Contains("faq.archive.org")
				|| url.Contains(".web.archive.org")
				|| url.Contains(".pdf")
				|| url.Contains(".doc")
				|| url.Contains(".ppt")
				|| url.Contains(".xls")
				|| url.Contains(".ps")
				|| url.Contains(".wmv")
				|| url.Contains("mediawiki.org")
				|| url.Contains("doubleclick")
				|| url.Contains("ads.")
				;
		}

		#region 1, 2, and 3 letter words
		private static string[] words1 = new string[] { "A", "I", "O" };
		private static string[] words2 = new string[] { "AA", "AB", "AD", "AE", "AG", "AH", "AI", "AL", "AM", "AN", "AR", "AS", "AT", "AW", "AX", "AY", "BA", "BE", "BI", "BO", "BY", "CH", "DA", "DE", "DI", "DO", "EA", "ED", "EE", "EF", "EH", "EL", "EM", "EN", "ER", 
				"ES", "ET", "EX+", "FA", "FE", "FY", "GI", "GO", "GU", "HA", "HE", "HI", "HM", "HO", "ID", "IF", "IN", "IO", "IS", "IT", "JA", "JO+", "KA", "KI", "KO", "KY", "LA", "LI", "LO", "MA", "ME", "MI", "MM", "MO", "MU", "MY", "NA", "NE", "NO", "NU", 
				"NY+", "OB", "OD", "OE", "OF", "OH", "OI", "OM", "ON", "OO", "OP", "OR", "OS+", "OU", "OW", "OX+", "OY", "PA", "PE", "PI", "PO", "QI", "RE", "SH", "SI", "SO", "ST", "TA  ", "TE", "TI", "TO", "UG", "UH", "UM", "UN", "UP", "UR", "US", "UT", 
				"WE", "WO", "XI", "XU", "YA", "YE", "YO", "YU", "ZA", "ZO" };
		private static string[] words3 = new string[] { "AAH", "AAL", "AAS", "ABA", "ABO", "ABS", "ABY", "ACE", "ACT", "ADD", "ADO", "ADS", "ADZ", "AFF", "AFT", "AGA", "AGE", "AGO", "AGS", "AHA", "AHI", "AHS", "AID", "AIL", "AIM", "AIN", "AIR", "AIS", "AIT", "ALA", 
				"ALB", "ALE", "ALL", "ALP", "ALS", "ALT", "AMA", "AMI", "AMP", "AMU", "ANA", "AND", "ANE", "ANI", "ANT", "ANY", "APE", "APO", "APP", "APT", "ARB", "ARC", "ARE", "ARF", "ARK", "ARM", "ARS", "ART", "ASH", "ASK", "ASP", "ASS", "ATE", "ATT", 
				"AUK", "AVA", "AVE", "AVO", "AWA", "AWE", "AWL", "AWN", "AXE", "AYE", "AYS", "AZO", "BAA", "BAD", "BAG", "BAH", "BAL", "BAM", "BAN", "BAP", "BAR", "BAS", "BAT", "BAY", "BED", "BEE", "BEG", "BEL", "BEN", "BES", "BET", "BEY", "BIB", "BID", 
				"BIG", "BIN", "BIO", "BIS", "BIT", "BIZ", "BOA", "BOB", "BOD", "BOG", "BOO", "BOP", "BOS", "BOT", "BOW", "BOX", "BOY", "BRA", "BRO", "BRR", "BUB", "BUD", "BUG", "BUM", "BUN", "BUR", "BUS", "BUT", "BUY", "BYE", "BYS", "CAB", "CAD", "CAM", 
				"CAN", "CAP", "CAR", "CAT", "CAW", "CAY", "CEE", "CEL", "CEP", "CHI", "CIG", "CIS", "COB", "COD", "COG", "COL", "CON", "COO", "COP", "COR", "COS", "COT", "COW", "COX", "COY", "COZ", "CRU", "CRY", "CUB", "CUD", "CUE", "CUM", "CUP", "CUR", 
				"CUT", "CWM", "DAB", "DAD", "DAG", "DAH", "DAK", "DAL", "DAM", "DAN", "DAP", "DAW", "DAY", "DEB", "DEE", "DEF", "DEL", "DEN", "DEV", "DEW", "DEX", "DEY", "DIB", "DID", "DIE", "DIF", "DIG", "DIM", "DIN", "DIP", "DIS", "DIT", "DOC", "DOE", 
				"DOG", "DOL", "DOM", "DON", "DOR", "DOS", "DOT", "DOW", "DRY", "DUB", "DUD", "DUE", "DUG", "DUH", "DUI", "DUN", "DUO", "DUP", "DYE", "EAR", "EAT", "EAU", "EBB", "ECU", "EDH", "EDS", "EEK", "EEL", "EFF", "EFS", "EFT", "EGG", "EGO", "EKE", 
				"ELD", "ELF", "ELK", "ELL", "ELM", "ELS", "EME", "EMS", "EMU", "END", "ENG", "ENS", "EON", "ERA", "ERE", "ERG", "ERN", "ERR", "ERS", "ESS", "ETA", "ETH", "EVE", "EWE", "EYE", "FAB", "FAD", "FAG", "FAN", "FAR", "FAS", "FAT", "FAX", "FAY", 
				"FED", "FEE", "FEH", "FEM", "FEN", "FER", "FES", "FET", "FEU", "FEW", "FEY", "FEZ", "FIB", "FID", "FIE", "FIG", "FIL", "FIN", "FIR", "FIT", "FIX", "FIZ", "FLU", "FLY", "FOB", "FOE", "FOG", "FOH", "FON", "FOP", "FOR", "FOU", "FOX", "FOY", 
				"FRO", "FRY", "FUB", "FUD", "FUG", "FUN", "FUR", "GAB", "GAD", "GAE", "GAG", "GAL", "GAM", "GAN", "GAP", "GAR", "GAS", "GAT", "GAY", "GED", "GEE", "GEL", "GEM", "GEN", "GET", "GEY", "GHI", "GIB", "GID", "GIE", "GIG", "GIN", "GIP", "GIT", 
				"GNU", "GOA", "GOB", "GOD", "GOO", "GOR", "GOS", "GOT", "GOX", "GOY", "GUL", "GUM", "GUN", "GUT", "GUV", "GUY", "GYM", "GYP", "HAD", "HAE", "HAG", "HAH", "HAJ", "HAM", "HAO", "HAP", "HAS", "HAT", "HAW", "HAY", "HEH", "HEM", "HEN", "HEP", 
				"HER", "HES", "HET", "HEW", "HEX", "HEY", "HIC", "HID", "HIE", "HIM", "HIN", "HIP", "HIS", "HIT", "HMM", "HOB", "HOD", "HOE", "HOG", "HON", "HOP", "HOT", "HOW", "HOY", "HUB", "HUE", "HUG", "HUH", "HUM", "HUN", "HUP", "HUT", "HYP", "ICE", 
				"ICH", "ICK", "ICY", "IDS", "IFF", "IFS", "IGG", "ILK", "ILL", "IMP", "INK", "INN", "INS", "ION", "IRE", "IRK", "ISM", "ITS", "IVY", "JAB", "JAG", "JAM", "JAR", "JAW", "JAY", "JEE", "JET", "JEU", "JEW", "JIB", "JIG", "JIN", "JOB", "JOE", 
				"JOG", "JOT", "JOW", "JOY", "JUG", "JUN", "JUS", "JUT", "KAB", "KAE", "KAF", "KAS", "KAT", "KAY", "KEA", "KEF", "KEG", "KEN", "KEP", "KEX", "KEY", "KHI", "KID", "KIF", "KIN", "KIP", "KIR", "KIS", "KIT", "KOA", "KOB", "KOI", "KOP", "KOR", 
				"KOS", "KUE", "KYE", "LAB", "LAC", "LAD", "LAG", "LAM", "LAP", "LAR", "LAS", "LAT", "LAV", "LAW", "LAX", "LAY", "LEA", "LED", "LEE", "LEG", "LEI", "LEK", "LET", "LEU", "LEV", "LEX", "LEY", "LEZ", "LIB", "LID", "LIE", "LIN", "LIP", "LIS", 
				"LIT", "LOB", "LOG", "LOO", "LOP", "LOT", "LOW", "LOX", "LUG", "LUM", "LUV", "LUX", "LYE", "MAC", "MAD", "MAE", "MAG", "MAN", "MAP", "MAR", "MAS", "MAT", "MAW", "MAX", "MAY", "MED", "MEG", "MEL", "MEM", "MEN", "MET", "MEW", "MHO", "MIB", 
				"MIC", "MID", "MIG", "MIL", "MIM", "MIR", "MIS", "MIX", "MOA", "MOB", "MOC", "MOD", "MOG", "MOL", "MOM", "MON", "MOO", "MOP", "MOR", "MOS", "MOT", "MOW", "MUD", "MUG", "MUM", "MUN", "MUS", "MUT", "MYC", "NAB", "NAE", "NAG", "NAH", "NAM", 
				"NAN", "NAP", "NAW", "NAY", "NEB", "NEE", "NEG", "NET", "NEW", "NIB", "NIL", "NIM", "NIP", "NIT", "NIX", "NOB", "NOD", "NOG", "NOH", "NOM", "NOO", "NOR", "NOS", "NOT", "NOW", "NTH", "NUB", "NUN", "NUS", "NUT", "OAF", "OAK", "OAR", "OAT", 
				"OBA", "OBE", "OBI", "OCA", "ODA", "ODD", "ODE", "ODS", "OES", "OFF", "OFT", "OHM", "OHO", "OHS", "OIL", "OKA", "OKE", "OLD", "OLE", "OMS", "ONE", "ONO", "ONS", "OOH", "OOT", "OPE", "OPS", "OPT", "ORA", "ORB", "ORC", "ORE", "ORS", "ORT", 
				"OSE", "OUD", "OUR", "OUT", "OVA", "OWE", "OWL", "OWN", "OXO", "OXY", "PAC", "PAD", "PAH", "PAL", "PAM", "PAN", "PAP", "PAR", "PAS", "PAT", "PAW", "PAX", "PAY", "PEA", "PEC", "PED", "PEE", "PEG", "PEH", "PEN", "PEP", "PER", "PES", "PET", 
				"PEW", "PHI", "PHT", "PIA", "PIC", "PIE", "PIG", "PIN", "PIP", "PIS", "PIT", "PIU", "PIX", "PLY", "POD", "POH", "POI", "POL", "POM", "POP", "POT", "POW", "POX", "PRO", "PRY", "PSI", "PST", "PUB", "PUD", "PUG", "PUL", "PUN", "PUP", "PUR", 
				"PUS", "PUT", "PYA", "PYE", "PYX", "QAT", "QIS", "QUA", "RAD", "RAG", "RAH", "RAI", "RAJ", "RAM", "RAN", "RAP", "RAS", "RAT", "RAW", "RAX", "RAY", "REB", "REC", "RED", "REE", "REF", "REG", "REI", "REM", "REP", "RES", "RET", "REV", "REX", 
				"RHO", "RIA", "RIB", "RID", "RIF", "RIG", "RIM", "RIN", "RIP", "ROB", "ROC", "ROD", "ROE", "ROM", "ROT", "ROW", "RUB", "RUE", "RUG", "RUM", "RUN", "RUT", "RYA", "RYE", "SAB", "SAC", "SAD", "SAE", "SAG", "SAL", "SAP", "SAT", "SAU", "SAW", 
				"SAX", "SAY", "SEA", "SEC", "SEE", "SEG", "SEI", "SEL", "SEN", "SER", "SET", "SEW", "SEX", "SHA", "SHE", "SHH", "SHY", "SIB", "SIC", "SIM", "SIN", "SIP", "SIR", "SIS", "SIT", "SIX", "SKA", "SKI", "SKY", "SLY", "SOB", "SOD", "SOL", "SOM", 
				"SON", "SOP", "SOS", "SOT", "SOU", "SOW", "SOX", "SOY", "SPA", "SPY", "SRI", "STY", "SUB", "SUE", "SUK", "SUM", "SUN", "SUP", "SUQ", "SYN", "TAB", "TAD", "TAE", "TAG", "TAJ", "TAM", "TAN", "TAO", "TAP", "TAR", "TAS", "TAT", "TAU", "TAV", 
				"TAW", "TAX", "TEA", "TED", "TEE", "TEG", "TEL", "TEN", "TET", "TEW", "THE", "THO", "THY", "TIC", "TIE", "TIL", "TIN", "TIP", "TIS", "TIT", "TOD", "TOE", "TOG", "TOM", "TON", "TOO", "TOP", "TOR", "TOT", "TOW", "TOY", "TRY", "TSK", "TUB", 
				"TUG", "TUI", "TUN", "TUP", "TUT", "TUX", "TWA", "TWO", "TYE", "UDO", "UGH", "UKE", "ULU", "UMM", "UMP", "UNS", "UPO", "UPS", "URB", "URD", "URN", "URP", "USE", "UTA", "UTE", "UTS", "VAC", "VAN", "VAR", "VAS", "VAT", "VAU", "VAV", "VAW", 
				"VEE", "VEG", "VET", "VEX", "VIA", "VID", "VIE", "VIG", "VIM", "VIS", "VOE", "VOW", "VOX", "VUG", "VUM", "WAB", "WAD", "WAE", "WAG", "WAN", "WAP", "WAR", "WAS", "WAT", "WAW", "WAX", "WAY", "WEB", "WED", "WEE", "WEN", "WET", "WHA", "WHO", 
				"WHY", "WIG", "WIN", "WIS", "WIT", "WIZ", "WOE", "WOG", "WOK", "WON", "WOO", "WOP", "WOS", "WOT", "WOW", "WRY", "WUD", "WYE", "WYN", "XIS", "YAG", "YAH", "YAK", "YAM", "YAP", "YAR", "YAW", "YAY", "YEA", "YEH", "YEN", "YEP", "YES", "YET", 
				"YEW", "YID", "YIN", "YIP", "YOB", "YOD", "YOK", "YOM", "YON", "YOU", "YOW", "YUK", "YUM", "YUP", "ZAG", "ZAP", "ZAS", "ZAX", "ZED", "ZEE", "ZEK", "ZEP", "ZIG", "ZIN", "ZIP", "ZIT", "ZOA", "ZOO", "ZUZ", "ZZZ" };
		#endregion 2 and 3 letter words
		#region Predefined word types

		private static string[] adjectives = new string[] { "costly" };

		private static string[] prepositions = new string[] { "abaft", "aboard", "about", "above", "absent", "across", "afore", "after", "against", "along", "alongside", "amid", "amidst", "among", "amongst", "an", "apropos", "around", "as", "aside", "astride", "at", 
			"athwart", "atop", "barring", "before", "behind", "below", "beneath", "beside", "besides", "between", "betwixt", "beyond", "but", "by", "circa", "concerning", "despite", "down", "during", "except", "excluding", "failing", "following", "for", 
			"from", "given", "in", "including", "inside", "into", "like", "mid", "midst", "minus", "modulo", "near", "next", "notwithstanding", "of", "off", "on", "onto", "opposite", "out", "outside", "over", "pace", "past", "per", "plus", "pro", "qua", 
			"regarding", "round", "sans", "save", "since", "than", "through", "throughout", "till", "times", "to", "toward", "towards", "under", "underneath", "unlike", "until", "up", "upon", "versus", "via", "vice", "with", "within", "without", "worth" };

		private static string[] articles = new string[] { "a", "an", "da", "teh", "the", "ye", "ze", "zee", "ðe" };

		private static string[] adverbs = new string[] { "so", "to", "because", "since", "accidentally", "intentionally", "purposely", "after", "already", "during", "finally", "just", "last", "later", "next", "now", "recently", "soon", "then", "tomorrow", "when", "while", 
			"yesterday", "always", "almost", "afterwards", "even", "never", "sometimes", "often", "not", "soon", "too", "very", "upward", "downward", "leftward", "righward", "eastbound", "northbound", "southbound", "westbound", "upbeat", "well", "upright" };

		private static string[] conjunctions = new string[] { "and", "or", "for", "nor", "but", "yet", "so", "either", "neither", "both", "whether", "after", "although", "as", "because", "before", "after", "both", "all", "how", "who", "what", "when", "where", "why",
			"however", "if", "nor", "now", "then", "once", "only", "rather", "since", "than", "that", "though", "til", "till", "until", "unless", "when", "whenever", "where", "whereas", "wherever", "whoever", "whatever", "whyever", "while", "yet" };

		#endregion Predefined word types
		#endregion Enums and constants



	}
}
